# Updated by Wei on 2014/01/25 afternoon at school
# generate the OR document candidate results
# Updated by Wei on 2014/01/21 morning at school
# How to evaluate the OR semantics? At least have some prelimanry results in order to make prof and Juan easy
# Updated by Wei on 2014/01/17 night at school
# Make the evaluation for the piece wise approach in c
# Updated by Wei on 2014/01/06 night at school
# Note: Make the evaluation consistent with the python simulation code (In progress)
# Line 163 need to change accordingly
# Updated by Wei on 2014/01/04 afternoon at school (do the relrank simulation)
# Updated by Wei on 2013/12/10 night at school
# Updated by Wei 2013/08/08 morning
# Note: sample command:
# python /data3/obukai/workspace/web-search-engine-wei/scripts/src/pythonScripts/gov2/step10_gov2_compute_symmetric_difference.py 
# /data3/obukai/workspace/web-search-engine-wei/polyIRIndexer/2013AugPruningResults/goldStandardResultsForPruningProject_AND_TOP10_tail5k 
# /data3/obukai/workspace/web-search-engine-wei/polyIRIndexer/2013AugPruningResults/goldStandardResultsForPruningProject_AND_TOP10_tail5k 
# NUM_OF_LINES_NEEDED_TO_SKIP : 0 
# NUM_OF_TOP_RESULTS_FOR_EVALUATION : 10 
# PRUNING_METHOD : uniformPartialBM25 
# PERCENTAGE_KEPT : 100 
# CONDITION : AND

# python /data3/obukai/workspace/web-search-engine-wei/scripts/src/pythonScripts/gov2/step10_gov2_compute_symmetric_difference.py /data3/obukai/workspace/web-search-engine-wei/polyIRIndexer/2013AugPruningResults/goldStandardResultsForPruningProject_AND_TOP10_tail5k_qid_95984_STOP /data3/obukai/workspace/web-search-engine-wei/polyIRIndexer/2013AugPruningResults/goldStandardResultsForPruningProject_AND_TOP10_tail5k_qid_95984_STOP 0 10 uniformPartialBM25 100 AND
from __future__ import division
import random
import math
import os
import sys
from sets import Set
# Updated by Wei 2013/08/04 night at school
# optimized for the 3 factor probablies formula

# Updated by Wei 20130621
# Basically this time, use this program in the machine pangolin to produce some results

# Updated by Wei 20130525
# some raw results produced by the irtk

# Program inputs:
# gold standard file
# inputFileName1 = ""

# compared file
# inputFileName2 = ""

# Program outputs:
# the symmetric_difference

def getQueryIDAndResultSetGivenResultFileForNewFormatResultFile(inputFileName,outputFileName1,outputFileName2,NUM_OF_TOP_RESULTS_NEEDED_TO_EVALUATE,NUM_OF_QUERIES_NEEDED_TO_EXTRACT):
    print "getQueryIDAndResultSetGivenResultFileForNewFormatResultFile() function begins..."
    numOfQueriesHavingQID = 0
    numOfQueriesHavingSearchContent = 0
    numOfQueriesHavingSearchResults = 0
    numOfQueriesHavingCompleteTOP10SearchResults = 0
    qidListWithoutResultsList = []
    qidListWithIncompletedTOP10ResultsList = []
    
    # key:queryID value:a list of tuples with the format (rank,docID)
    insideQueryIDAndResultSetDict = {}
    QIDAndLowerBoundDict = {}
    
    inputFileHandler = open(inputFileName,"r")
    outputFileHandler1 = open(outputFileName1,"w")
    outputFileHandler2 = open(outputFileName2,"w")
    
    currentLine = inputFileHandler.readline()
    nextLine = ""
    currentQID = 0
    
    numOfResultsForTheAllQueries = 0
    
    while currentLine:
        # sampling parsing line:
        # qid: 701
        if currentLine.startswith("qid:"):
            currentQID = int( currentLine.strip().split(":")[1].strip() )
            print "currentQID:",str( currentQID )
            
            if numOfQueriesHavingQID == NUM_OF_QUERIES_NEEDED_TO_EXTRACT:
                break
            
            if currentQID not in insideQueryIDAndResultSetDict:
                insideQueryIDAndResultSetDict[currentQID] = Set([])
            numOfQueriesHavingQID += 1
        
        # sample parsing line:
        # Search: u s  oil industry history
        if currentLine.startswith("Search:"):
            elementList = currentLine.strip().split(" ")
            
            currentSearchContent = ""
            
            queryTermList = []
            for element in elementList[1:]:
                if element.strip() != "":
                    queryTermList.append( element.strip() )        
            currentQueryLength = len(queryTermList)
            print "currentQueryLength:",currentQueryLength
    
            for element in queryTermList:
                currentSearchContent += element + " "
            
            print currentSearchContent
            numOfQueriesHavingSearchContent += 1
            
            
            currentQueryTermIndexDict = {}
            nextLine = inputFileHandler.readline()
            #print nextLine.strip()
            parsedFlag = True
            skipOneLineFlag = False
            
            if nextLine.strip().endswith("is NOT in the lexicon."):
                # This is for the parsing of OR semantics
                if currentQueryLength == 1: # That means the only query term is NOT in the lexicon
                    parsedFlag = False
                    qidListWithoutResultsList.append(currentQID)
                else:
                    parsedFlag = True
                    skipOneLineFlag = True
                # This is for the parsing of the AND semantics
                # parsedFlag = False 
            
            if nextLine.strip() != "" and parsedFlag:
                
                if skipOneLineFlag:
                    # skip the line ends with "is NOT in the lexicon"
                    nextLine = inputFileHandler.readline()
                nextLineElements = nextLine.strip().split(" ")
                # print "nextLineElements:",nextLineElements
                
                # sample parsing line: 
                # oil:0 industry:1 history:2 u:3 s:4
                for element in nextLineElements:
                    if element.split(":")[0] not in currentQueryTermIndexDict:
                        currentQueryTermIndexDict[ int(element.split(":")[1]) ] = element.split(":")[0]
                
                print "currentQueryTermIndexDict:",currentQueryTermIndexDict
                # print "len(currentQueryTermIndexDict):",len(currentQueryTermIndexDict)
                
                # Now, it is time to read the threshold line
                currentLine = inputFileHandler.readline()
                if len(currentLine.strip().split(":")) == 2:
                    name = currentLine.strip().split(":")[0]
                    value = currentLine.strip().split(":")[1]
                    print name,"::",value
                else:
                    print "format problem"
                    exit(1)
                    
                # Now, it is time to read computation method line
                currentLine = inputFileHandler.readline()
                if len(currentLine.strip().split(":")) == 2:
                    name = currentLine.strip().split(":")[0]
                    value = currentLine.strip().split(":")[1]
                    print name,"::",value
                else:
                    print "format problem"
                    exit(1)
                
                # skip the line like the following:
                # 3 factor probability formula logic implemented by Wei on 2013/08/02 at school    
                currentLine = inputFileHandler.readline()
                # currentLine = inputFileHandler.readline()
                
                # Now, it is time to read the actual training example line
                currentLine = inputFileHandler.readline()
                numOfResultsForTheCurrentQuery = 0
                currentResultLines = []
                # for DEBUG ONLY
                # print "len( currentLine.strip().split(" ") ):",len( currentLine.strip().split(" ") )
                # print "currentLine:",currentLine
                # exit(1)
                
                lineElements = currentLine.strip().split(" ")
                
                
                # CURRENT result format version
                while len( lineElements ) == 23:
                    base = 1
                    theRank = int(lineElements[0])
                    theDocID = lineElements[-1]
                    totalValue = float(lineElements[-2])
                    if theRank <= NUM_OF_TOP_RESULTS_NEEDED_TO_EVALUATE:
                        upperBound = 0
                        if len(currentQueryTermIndexDict) <= 10:
                            upperBound = len(currentQueryTermIndexDict)
                        else:
                            upperBound = 10
                        specialValue = upperBound - 1
                        for i in range(0,upperBound):
                            outputLine = str(currentQID) + " " + str(theDocID) + " " + str(currentQueryTermIndexDict[i]) + " " + str(lineElements[base+i]) + "\n"
                            outputFileHandler1.write(outputLine)
                        
                        QIDAndLowerBoundDict[currentQID] = totalValue
                        numOfResultsForTheCurrentQuery += 1
                        currentResultLines.append(currentLine.strip())
                    currentLine = inputFileHandler.readline()
                    lineElements = currentLine.strip().split(" ")
                
                numOfResultsForTheAllQueries += numOfResultsForTheCurrentQuery
                
                # sampling method 1: among all the top10 documents, select those documents, open those documents and extract the corresponding postings.
                # This method make sure that once the document have been selected, ALL the coresponding postings will also be selected.
                # This sampling method has been developed on 2013/01/09
                
                print "numOfResultsForTheCurrentQuery:",numOfResultsForTheCurrentQuery
                print "len(currentResultLines):",len(currentResultLines)
                if len(currentResultLines) != 0:
                    numOfQueriesHavingSearchResults += 1
                if len(currentResultLines) == 0:
                    qidListWithoutResultsList.append(currentQID)
                    
                if len(currentResultLines) == 10:
                    numOfQueriesHavingCompleteTOP10SearchResults += 1
                if len(currentResultLines) > 0 and len(currentResultLines) < 10:
                    qidListWithIncompletedTOP10ResultsList.append(currentQID)
                
                print
        currentLine = inputFileHandler.readline()
        
    for qid in QIDAndLowerBoundDict:
        outputFileHandler2.write(str(qid) + " " + str(QIDAndLowerBoundDict[qid]) + "\n")
    print "numOfQueriesHavingQID:",numOfQueriesHavingQID
    print "numOfQueriesHavingSearchContent:",numOfQueriesHavingSearchContent
    print "numOfQueriesHavingSearchResults:",numOfQueriesHavingSearchResults
    print "len(insideQueryIDAndResultSetDict):",len(insideQueryIDAndResultSetDict)
    print "numOfQueriesHavingCompleteTOP10SearchResults:",numOfQueriesHavingCompleteTOP10SearchResults
    print "qidListWithIncompletedTOP10ResultsList:",qidListWithIncompletedTOP10ResultsList
    print "qidListWithoutResultsList:",qidListWithoutResultsList
    # print "insideQueryIDAndResultSetDict[95085]:",insideQueryIDAndResultSetDict[95085]
    # print "insideQueryIDAndResultSetDict[95086]:",insideQueryIDAndResultSetDict[95086]
    print "getQueryIDAndResultSetGivenResultFileForNewFormatResultFile() function ends."
    print "Overall:"
    print "inputFileName:",inputFileName
    print "outputFileName1:",outputFileName1
    print "outputFileName2:",outputFileName2
    inputFileHandler.close()
    outputFileHandler1.close()
    outputFileHandler2.close()
    return insideQueryIDAndResultSetDict

print "Program Begins..."
inputFileName = "/home/diaosi/web-search-engine-wei_MOVE_FROM_PANGOLIN_20131206/polyIRIndexer/raw_100%_tail5KResults_OR_semantics"
outputFileName1 = "/home/diaosi/web-search-engine-wei_MOVE_FROM_PANGOLIN_20131206/polyIRIndexer/tail5KResults_NEW_FORMAT_20140222Afternoon_middle_step1"
outputFileName2 = "/home/diaosi/web-search-engine-wei_MOVE_FROM_PANGOLIN_20131206/polyIRIndexer/tail5KResults_20140222Night_lowerBoundThresholds"
TOPKValue = 10
NUM_OF_QUERIES_NEEDED_TO_EXTRACT = 10000
getQueryIDAndResultSetGivenResultFileForNewFormatResultFile(inputFileName,outputFileName1,outputFileName2,TOPKValue,NUM_OF_QUERIES_NEEDED_TO_EXTRACT)
print "Program Ends."

















